require 'jcode'
class HotPost::TianYa < HotPost


  def after_initialize
    if new_record? and requested_url.present?
      logger.debug "开始访问#{self.requested_url}"
      begin
        # 请求的url
        self.requested_url = self.requested_url.strip
        # 首页 url
        self.first_page_url = first_page_url_from(requested_url)
        logger.debug "first_page_url: #{first_page_url}"
        # 标题
        power_page_docs_up [first_page_doc], :from_page_num => 1, :only_first => true
        self.title = first_page_doc.title
        rename_title!
        # 作者
        self.author = first_page_doc.first_author
        # 分类
        self.hot_post_type = HotPostType::TianYa.find_or_create_by_name(first_page_doc.type_name)
        # 发布时间
        self.first_released_at = first_page_doc.first_released_at
      rescue URI::InvalidURIError => e
        logger.error "url格式不对：#{e}"
        @requested_url_error = '您输入的网址格式有问题，请重新复制、粘贴一次'
      rescue OpenURI::HTTPError => e
        logger.error "天涯社区拒绝：#{e}"
        @requested_url_error = '该网址对应的页面似乎不存在，再试试？'
      rescue SocketError => e
        logger.error "麦吐司网暂时访问不了天涯社区：#{e}"
        @requested_url_error = '麦吐司网暂时访问不了天涯社区,请稍后重试'
      rescue RuntimeError => e
        if e.to_s == 'unknown style'
          logger.error "该网址对应的页面不是天涯社区的帖：#{e}"
          @requested_url_error = '该网址对应的页面不是天涯社区的帖,无法追它'
        else
          raise e
        end
      end
    end
  end
  
  def requested_url_error?
    @requested_url_error
  end
  
  def requested_url_error
    @requested_url_error
  end
  
  def first_post_content
    first_page_doc.posts.first
  end

  def page_docs(options = { :from_page_num => 1,:limit => 50 })
    if options[:from_page_num] == 1
      debug "page_docs() from_page_num_1 limit is #{options[:limit]}"
      options[:limit] -= 1
      @all_pages_docs ||= [first_page_doc] + no_first_page_docs(options)
      debug "all_pages_docs"
      power_page_docs_up @all_pages_docs,options
    else
      debug "page_docs() from_page_num_#{options[:from_page_num]} limit is #{options[:limit]}"
      @sub_page_docs_hash ||= {}
      if @sub_page_docs_hash[options[:from_page_num]].nil?
        @sub_page_docs_hash[options[:from_page_num]] = []
        @sub_page_docs_hash[options[:from_page_num]] = urls_to_docs(page_urls(options))
        debug "power_page_docs_up from_page_num_#{options[:from_page_num]} 's size is #{@sub_page_docs_hash[options[:from_page_num]].size}, :limit => #{options[:limit]}"
        power_page_docs_up @sub_page_docs_hash[options[:from_page_num]], options
      else
        @sub_page_docs_hash[options[:from_page_num]]
      end
    end
  end

  def page_urls(options = { :from_page_num => 1,:limit => 50})
    @page_urls ||= {}
    debug "page_urls(:from_page_num => #{options[:from_page_num]}, :limit => #{options[:limit]})"
    if options[:from_page_num] == 1
      debug "start new_one"
      @page_urls[options[:from_page_num]] ||= [first_page_url] + no_first_page_urls(first_page_doc,options)
    else
      debug "continue from url:#{the_last_sub_hot_post_by_tian_ya_page_num(options[:from_page_num]).url}"
      @page_urls[options[:from_page_num]] ||= [the_last_sub_hot_post_by_tian_ya_page_num(options[:from_page_num]).url] + no_first_page_urls(url_to_doc(the_last_sub_hot_post_by_tian_ya_page_num(options[:from_page_num]).url),options)
    end
  end

  # 抓取
  def capture!(options = { :from_page_num => 1,:limit => 50})
    if new_record?
      debug "新帖"
      page_docs(options).each_with_object(self.sub_hot_posts) do |doc, sub_posts|
        do_if_capture_per_page_sub_hot_posts_exception do 
          doc.sub_hot_posts(self).each do |sub_post|
            sub_posts << sub_post
          end
        end
      end
    else
      debug "旧贴 #{the_last_sub_hot_post.try(:page_num)} 页"
#      debug "the_last_sub_hot_post :page_num => #{the_last_sub_hot_post.page_num}, :position => #{the_last_sub_hot_post.position}"
      options[:from_page_num] = the_last_sub_hot_post.try(:page_num) || 1
      page_docs(options).each_with_object(the_last_sub_hot_post ? [the_last_sub_hot_post] : []) do |doc, sub_posts|
        debug "====capturing_page_#{doc.page_num}: #{doc.page_url}"
        do_if_capture_per_page_sub_hot_posts_exception do 
          doc.sub_hot_posts(self).each do |sub_post|
            debug "sub_posts.size is #{sub_posts.size}"
            if sub_posts.empty?
              debug "sub_posts is empty, so sub_post[#{sub_post.page_num}-#{sub_post.position}] >> "
              sub_posts << sub_post
            elsif sub_post.position > sub_posts.last.position and sub_post.page_num == sub_posts.last.page_num
              debug "page_#{sub_post.page_num} continue, so sub_post[#{sub_post.page_num}-#{sub_post.position}] >> "
              sub_posts << sub_post
            elsif sub_post.page_num > sub_hot_posts.last.page_num
              debug "new page#{sub_post.page_num}, so sub_post[#{sub_post.page_num}-#{sub_post.position}] >> "
              sub_posts << sub_post
            else
#              sub_post.readonly!
              debug 'do nothing'
            end
            debug "sub_post.released_at is #{sub_post.released_at}"
          end
          sub_posts.each do |sub_post|
            debug(sub_post.save(false) ? "saved sub_post[#{sub_post.page_num}-#{sub_post.position}-#{sub_post.released_at}]" : "error: sub_post(:page_num => #{sub_post.page_num}, :postion => #{sub_post.position}) #{sub_post.errors.full_messages}") if sub_post.new_record?
          end
        end
      end
    end
    reset_stat!
  end
  
  def rename_title!
    power_page_docs_up [first_page_doc], :from_page_num => 1, :only_first => true
    self.title = first_page_doc.title
  end

  private

  def power_page_docs_up(page_docs, options = { :from_page_num => 1, :only_first => false, :limit => 50 })
    options[:only_first] ||= false
    debug "power up starting ... :limit => #{options[:limit]} "
    page_docs.each_with_index do |page_doc,index|      
      def page_doc.init(args = {})
        @page_docs = args[:page_docs]
        @page_urls = args[:page_urls]
        # style
        if first_page_doc.css('table#firstAuthor > tr > td > a').any?
          @code_style = 'style1'
        elsif first_page_doc.css('div#pContentDiv > div.item > div.vcard > a').any?
          @code_style = 'style2'
        else
          raise 'unknown style'
        end
      end

      def page_doc.style1?
        @code_style == 'style1'
      end

      def page_doc.style2?
        @code_style == 'style2'
      end

      def page_doc.type_name
        # style1 && style2
        first_page_doc.css("div.pagewrap > h1 > span > a").text
      end

      def page_doc.title
        # style1 && style2
        #first_page_doc.css('meta[@name="Description"]').attr('content').value
        #first_page_doc.css('title').text
        first_page_doc.css("div.pagewrap > h1 > span").text.split('』')[1].strip
      end

      def page_doc.first_page_doc
        page_docs.first
      end

      def page_doc.first_author
        if style1?
          # style1
          css('table#firstAuthor > tr > td > a').first.text
        elsif style2?
          # style2
          css('div#pContentDiv > div.item > div.vcard > a').first.text
        else
          raise "unknown tian_ya code's style"
        end
      end

      def page_doc.first_released_at
        if style1?
          # style1
          css('table#firstAuthor > tr > td').text.split('发表日期：')[1].gsub(/  /,' ')
        elsif style2?
          # style2
          css('div#pContentDiv > div.item > div.vcard').first.text.split(first_page_doc == self ? '发表日期：':'回复日期：')[1].gsub('  ',' ')
        else
          raise "unknown tian_ya code's style"
        end
      end

      def page_doc.released_ats
        if style1?
          @released_ats ||= (
            # style1
            css('div#pContentDiv > table[@bgcolor="#f5f9fa"] > tr > td > center > font[@color="green"]') +
            css('div#pContentDiv > table[@bgcolor="f5f9fa"] > tr > td > center > font[@color="green"]')
          ).each_with_object([first_released_at]) do |node, released_ats|
#            node_text = Iconv.iconv("UTF-8//IGNORE","GBK//IGNORE",node.inner_html).to_s
#            puts "=#{page_url}============================#{node_text.split('回复日期：')[1]}=#{node_text}"
            released_ats << node.text.split('回复日期：')[1]
          end
        elsif style2?
          @released_ats ||= css('div#pContentDiv > div.item > div.vcard').each_with_object([]) do |node, released_ats|
            released_ats << node.text.split('日期：')[1]
          end
        else
          raise "unknown tian_ya code's style"
        end
      end

      def page_doc.page_docs
        @page_docs
      end

      def page_doc.page_urls
        @page_urls
      end     

      def page_doc.authors
        @authors ||= (
          # style1
          css('div#pContentDiv > table[@bgcolor="#f5f9fa"] > tr > td > center > font[@color="green"] > a') +
          css('div#pContentDiv > table[@bgcolor="f5f9fa"] > tr > td > center > font[@color="green"] > a') +
          # style2
          css('div#pContentDiv > div.item > div.vcard > a')
        ).each_with_object(style1? ? [first_author]:[]) do |author,authors|
          authors << author.text
        end
      end

      def page_doc.posts
        @posts ||= css('div.post').each_with_object([]) do |post,posts|
          p = Iconv.iconv("UTF-8//IGNORE","GBK//IGNORE",post.inner_html)
          def p.jlength=(jlength = 0)
            @jlength = jlength
          end
          def p.jlength
            @jlength
          end
          p.jlength = post.text.jlength
          posts << p.to_s
        end
      end

      def page_doc.sub_hot_posts(hot_post = nil)
        puts "==> page_#{page_num} authors[#{authors.size}] posts[#{posts.size}] "
        raise "
                authors' size(#{authors.size}) != posts' size(#{posts.size}) in page【#{page_num}】: #{page_url} 
                <div style='width:100%;'>
                #{  (
                    authors.size > posts.size ? (
                      authors.each_with_object([]) do |author, author_and_posts|
#                        puts "author_and_posts.size is #{author_and_posts.size}"
#                        puts "author is #{author}"
#                        puts "released_ats[author_and_posts.size] is #{released_ats[author_and_posts.size]}"
#                        puts "posts[author_and_posts.size] is #{posts[author_and_posts.size]}"
                        author_and_posts << " no.#{author_and_posts.size}【#{author}】- [#{released_ats[author_and_posts.size]}]:【#{posts[author_and_posts.size]}】"
                      end
                    ) : ( 
                      posts.each_with_object([]) do |post, author_and_posts|
#                        puts "author_and_posts.size is #{author_and_posts.size}"
#                        puts "authors[author_and_posts.size] is #{authors[author_and_posts.size]}"
#                        puts "post is #{post}"
                        author_and_posts << " no.#{author_and_posts.size}【#{authors[author_and_posts.size]}】:【#{post}】"
                      end
                    )
                ).join("</div><div style='width:100%;'>")
                
                }
              </div>
              " if authors.size != posts.size
        raise "authors' size(#{authors.size}) != released_ats' size(#{released_ats.size}) in page【#{page_num}】: #{page_url} 
                <div style='width:100%;'>
                #{  (
                    authors.size > posts.size ? (
                      authors.each_with_object([]) do |author, author_and_ats|
                        author_and_ats << " no.#{author_and_ats.size}【#{author}】:【#{released_ats[author_and_ats.size]}】"
                      end
                    ) : ( 
                      posts.each_with_object([]) do |post, author_and_ats|
                        author_and_ats << " no.#{author_and_ats.size}【#{authors[author_and_ats.size]}】:【#{post}】"
                      end
                    )
                ).join("</div><div style='width:100%;'>")
              } </div>
        " if authors.size != released_ats.size
        authors.each_with_object([]) do |author,sub_hot_posts|
          sub_hot_posts << SubHotPost.new(
            :hot_post => hot_post,
            :author => author,
            :content => posts[sub_hot_posts.size],
            :words_count => posts[sub_hot_posts.size].jlength,
            :released_at => released_ats[sub_hot_posts.size],
            :url => page_url,
            :page_num => page_num,
            :position => sub_hot_posts.size
          )
          #puts "=#{sub_hot_posts.last.released_at.to_s}===============================#{released_ats[sub_hot_posts.size].to_s}"
        end
      end
      
      if options[:only_first]
        page_doc.init( :page_docs => page_docs, :page_urls => [first_page_url])
      else
        page_doc.init( :page_docs => page_docs, :page_urls => page_urls(options))
      end
      
      debug "#{index}. power_up page_#{page_doc.page_num}"
    end
  end

  def no_first_page_docs(options = {:limit => 50})
#    first_page_doc.css('div#pageDivTop > a').each_with_object([]) do |link,docs|
##      debug "page #{link.text} - #{link['href']}"
#      docs << url_to_doc(link['href'])
#    end
#    raise '严重性能问题'
#    @no_first_page_docs ||= no_first_page_urls.each_with_object([]) do |url, docs|
#      debug "created doc from url_#{url}"
#      docs << url_to_doc(url)
#    end
    debug "no_first_page_docs(:limit => #{options[:limit]})"
    @no_first_page_docs ||= urls_to_docs(no_first_page_urls(first_page_doc,options))
  end

  def first_page_doc
    @first_page_doc ||=  url_to_doc(first_page_url)
  end

  def no_first_page_urls(doc = first_page_doc, options = {:limit => 50})
    debug "no_first_page_urls(:limit => #{options[:limit]})"
    @no_first_page_urls ||= []
    debug "@no_first_page_urls.size is #{@no_first_page_urls.size}"
#    debug "doc.css('div#pageDivTop > a') is #{doc.css('div#pageDivTop > a').size} "
#    debug "doc.css('div#cttPageDiv1 > form#pageForm > a') is #{doc.css('div#cttPageDiv1 > form#pageForm > a').size}"
#    debug "doc.css('form#pageForm > a') is #{doc.css('form#pageForm > a').size}"
#    debug "doc.css('div.function > div#cttPageDiv1 > a') is #{doc.css('div.function > div#cttPageDiv1 > a').size}"
    links = doc.css('div#pageDivTop > a') + doc.css('form#pageForm > a')
    debug " page links is #{links.size}"
    if links.any? and (not @no_first_page_urls.include?(links.last['href'])) and @no_first_page_urls.size < options[:limit]
      links.each do |link|
        if( not (@no_first_page_urls.include?(link['href']) or
            ['首页','上一页','下一页','末页','回复此贴'].include? link.text ) and @no_first_page_urls.size < options[:limit] 
         ) and (
           link.text.to_i > doc.page_num
         )
          debug "page #{link.text} - #{link['href']}"
          @no_first_page_urls << link['href']
        end
      end
      (@no_first_page_urls.any? and url_to_doc(@no_first_page_urls.last) != doc) ? 
        no_first_page_urls(url_to_doc(@no_first_page_urls.last),options) : @no_first_page_urls
    else
      @no_first_page_urls
    end
  end

  def first_page_url_from(url)
    doc = url_to_doc(url)
    current_page_em = doc.css('div#pageDivTop > em.current')
    page_links = doc.css('div#pageDivTop > a')
    (current_page_em.nil? or current_page_em.text == '1' or page_links.empty?) ? url : page_links.first['href']
  end

end
